<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html 
     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
  <title>Class: Crawler::Crawler</title>
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  <meta http-equiv="Content-Script-Type" content="text/javascript" />
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
  <script type="text/javascript">
  // <![CDATA[

  function popupCode( url ) {
    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
  }

  function toggleCode( id ) {
    if ( document.getElementById )
      elem = document.getElementById( id );
    else if ( document.all )
      elem = eval( "document.all." + id );
    else
      return false;

    elemStyle = elem.style;
    
    if ( elemStyle.display != "block" ) {
      elemStyle.display = "block"
    } else {
      elemStyle.display = "none"
    }

    return true;
  }
  
  // Make codeblocks hidden by default
  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
  
  // ]]>
  </script>

</head>
<body>



    <div id="classHeader">
        <table class="header-table">
        <tr class="top-aligned-row">
          <td><strong>Class</strong></td>
          <td class="class-name-in-header">Crawler::Crawler</td>
        </tr>
        <tr class="top-aligned-row">
            <td><strong>In:</strong></td>
            <td>
                <a href="../../files/lib/crawler/crawler_rb.html">
                lib/crawler/crawler.rb
                </a>
        <br />
            </td>
        </tr>

        <tr class="top-aligned-row">
            <td><strong>Parent:</strong></td>
            <td>
                Object
            </td>
        </tr>
        </table>
    </div>
  <!-- banner header -->

  <div id="bodyContent">



  <div id="contextContent">

    <div id="description">
      <p>
This is an abstrac crawler. It&#8216;s has a base structure to parse a
stream of services info and the crawl from the web the necessary
informayons
</p>

    </div>


   </div>

    <div id="method-list">
      <h3 class="section-bar">Methods</h3>

      <div class="name-list">
      <a href="#M000027">conditions</a>&nbsp;&nbsp;
      <a href="#M000023">new</a>&nbsp;&nbsp;
      <a href="#M000024">start_crawling</a>&nbsp;&nbsp;
      <a href="#M000026">stop_crawling?</a>&nbsp;&nbsp;
      <a href="#M000025">update_urls</a>&nbsp;&nbsp;
      </div>
    </div>

  </div>


    <!-- if includes -->

    <div id="section">





      


    <!-- if method_list -->
    <div id="methods">
      <h3 class="section-bar">Public Class methods</h3>

      <div id="method-M000023" class="method-detail">
        <a name="M000023"></a>

        <div class="method-heading">
          <a href="#M000023" class="method-signature">
          <span class="method-name">new</span><span class="method-args">(options = {})</span>
          </a>
        </div>
      
        <div class="method-description">
          <p>
instance the crawler, the options avaleable are:
</p>
<ul>
<li><b>:<a href="Crawler.html#M000027">conditions</a></b> the SQL <a
href="Crawler.html#M000027">conditions</a> that must be verify to retrive a
cawl from the web. <em>Default &quot;status is null or status &lt;&gt;
200&quot;</em>

</li>
<li><b>:<a href="Crawler.html#M000026">stop_crawling?</a></b> is a proedure
lambda {|crwal| return <em>[false|true]</em>} <em>Default lambda {|c|
false}</em>

</li>
</ul>
          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000023-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000023-source">
<pre>
    <span class="ruby-comment cmt"># File lib/crawler/crawler.rb, line 18</span>
18:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">options</span> = {})
19:       <span class="ruby-ivar">@curl</span> = <span class="ruby-constant">Curl</span><span class="ruby-operator">::</span><span class="ruby-constant">Easy</span>.<span class="ruby-identifier">new</span>
20:       
21:       <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">options</span>[<span class="ruby-identifier">:conditions</span>].<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">then</span>
22:         <span class="ruby-ivar">@conditions</span> = <span class="ruby-value str">&quot;status is null or status &lt;&gt; 200&quot;</span>
23:       <span class="ruby-keyword kw">else</span>
24:         <span class="ruby-ivar">@conditions</span> = <span class="ruby-identifier">options</span>[<span class="ruby-identifier">:conditions</span>]
25:       <span class="ruby-keyword kw">end</span>
26:       
27:       <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">options</span>[<span class="ruby-identifier">:stop_crawling?</span>].<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">then</span>
28:         <span class="ruby-ivar">@stop_crawling</span> = <span class="ruby-identifier">lambda</span> {<span class="ruby-operator">|</span><span class="ruby-identifier">c</span><span class="ruby-operator">|</span> <span class="ruby-keyword kw">false</span>}
29:       <span class="ruby-keyword kw">else</span>
30:         <span class="ruby-ivar">@stop_crawling</span> = <span class="ruby-identifier">options</span>[<span class="ruby-identifier">:stop_crawling?</span>]
31:       <span class="ruby-keyword kw">end</span>
32:     <span class="ruby-keyword kw">end</span>
</pre>
          </div>
        </div>
      </div>

      <h3 class="section-bar">Public Instance methods</h3>

      <div id="method-M000027" class="method-detail">
        <a name="M000027"></a>

        <div class="method-heading">
          <a href="#M000027" class="method-signature">
          <span class="method-name">conditions</span><span class="method-args">()</span>
          </a>
        </div>
      
        <div class="method-description">
          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000027-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000027-source">
<pre>
    <span class="ruby-comment cmt"># File lib/crawler/crawler.rb, line 70</span>
70:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">conditions</span>
71:       <span class="ruby-ivar">@conditions</span>
72:     <span class="ruby-keyword kw">end</span>
</pre>
          </div>
        </div>
      </div>

      <div id="method-M000024" class="method-detail">
        <a name="M000024"></a>

        <div class="method-heading">
          <a href="#M000024" class="method-signature">
          <span class="method-name">start_crawling</span><span class="method-args">() {|c if block| ...}</span>
          </a>
        </div>
      
        <div class="method-description">
          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000024-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000024-source">
<pre>
    <span class="ruby-comment cmt"># File lib/crawler/crawler.rb, line 34</span>
34:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">start_crawling</span> <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>
35:       <span class="ruby-identifier">log</span> = {<span class="ruby-identifier">:crwaled_success</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-value">0</span>, <span class="ruby-identifier">:crwaled_faild</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-value">0</span>, <span class="ruby-identifier">:stopped</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-keyword kw">false</span>}
36:       <span class="ruby-identifier">crawl_table</span>.<span class="ruby-identifier">find_each</span>(<span class="ruby-identifier">:conditions</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-identifier">conditions</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">c</span><span class="ruby-operator">|</span>
37:         <span class="ruby-identifier">response</span> = <span class="ruby-identifier">crawl</span>(<span class="ruby-identifier">c</span>)    
38:         
39:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">response</span> <span class="ruby-keyword kw">then</span>
40:           <span class="ruby-identifier">log</span>[<span class="ruby-identifier">:crwaled_success</span>] <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
41:         <span class="ruby-keyword kw">else</span>
42:           <span class="ruby-identifier">log</span>[<span class="ruby-identifier">:crwaled_faild</span>] <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
43:         <span class="ruby-keyword kw">end</span>
44:         
45:         <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">c</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block</span>    
46:         
47:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">stop_crawling?</span>(<span class="ruby-identifier">c</span>) <span class="ruby-keyword kw">then</span>
48:           <span class="ruby-identifier">log</span>[<span class="ruby-identifier">:stopped</span>] = <span class="ruby-keyword kw">true</span>
49:           <span class="ruby-keyword kw">break</span>
50:         <span class="ruby-keyword kw">end</span>
51:       }
52:       <span class="ruby-identifier">log</span>
53:     <span class="ruby-keyword kw">end</span>
</pre>
          </div>
        </div>
      </div>

      <div id="method-M000026" class="method-detail">
        <a name="M000026"></a>

        <div class="method-heading">
          <a href="#M000026" class="method-signature">
          <span class="method-name">stop_crawling?</span><span class="method-args">(r)</span>
          </a>
        </div>
      
        <div class="method-description">
          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000026-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000026-source">
<pre>
    <span class="ruby-comment cmt"># File lib/crawler/crawler.rb, line 66</span>
66:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stop_crawling?</span>(<span class="ruby-identifier">r</span>)
67:       <span class="ruby-ivar">@stop_crawling</span>.<span class="ruby-identifier">call</span>(<span class="ruby-identifier">r</span>)
68:     <span class="ruby-keyword kw">end</span>
</pre>
          </div>
        </div>
      </div>

      <div id="method-M000025" class="method-detail">
        <a name="M000025"></a>

        <div class="method-heading">
          <a href="#M000025" class="method-signature">
          <span class="method-name">update_urls</span><span class="method-args">()</span>
          </a>
        </div>
      
        <div class="method-description">
          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000025-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000025-source">
<pre>
    <span class="ruby-comment cmt"># File lib/crawler/crawler.rb, line 55</span>
55:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">update_urls</span>
56:       <span class="ruby-identifier">crawled</span> = <span class="ruby-value">0</span>
57:       <span class="ruby-constant">Service</span>.<span class="ruby-identifier">find_each</span>(<span class="ruby-identifier">:conditions</span> =<span class="ruby-operator">&gt;</span> [<span class="ruby-value str">'ff_id = ?'</span>, <span class="ruby-identifier">service_name</span>]) <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">s</span><span class="ruby-operator">|</span>
58:         <span class="ruby-keyword kw">if</span> <span class="ruby-keyword kw">not</span> <span class="ruby-identifier">s</span>.<span class="ruby-identifier">crawled?</span> <span class="ruby-keyword kw">then</span>
59:           <span class="ruby-identifier">crawl_table</span>.<span class="ruby-identifier">create</span> <span class="ruby-identifier">:service_id</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-identifier">s</span>.<span class="ruby-identifier">id</span>
60:           <span class="ruby-identifier">crawled</span> <span class="ruby-operator">+=</span><span class="ruby-value">1</span>
61:         <span class="ruby-keyword kw">end</span>
62:       <span class="ruby-keyword kw">end</span>
63:       <span class="ruby-identifier">crawled</span>
64:     <span class="ruby-keyword kw">end</span>
</pre>
          </div>
        </div>
      </div>


    </div>


  </div>


<div id="validator-badges">
  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
</div>

</body>
</html>